#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2021/8/9 10:48
# @Author  : Samge
import datetime

from itkz.spider_ext.items.itkz import ItkzItem
from itkz.spiders.base import BaseCommonSpider
from itkz.resources.utils import str_util, settings_util

null = None
false = False
true = True


class ItkzSpider(BaseCommonSpider):
    name = "ItkzSpider"

    DEFAULT_START_PAGE = 1
    IS_RESTART = True
    BATCH = 1

    # 需要置空的的关键词
    IGNORE_KEYS = [
        '【IT客栈 www.itkz.net】',
        '【IT客栈www.itkz.net】',
        '【www.ikz.net】',
        '【 www.ikz.net】',
        '【 www.ikz.ne】',
        'www.ikz.ne',
        'www.itkz.net',
        'itkz.net',
        'itkz',
        'IT客栈',
        '客栈',
        '【 】',
        '【】',
    ]

    custom_settings = {
        'ITEM_PIPELINES': {'itkz.spider_ext.pipelines.itkz.ItkzDb': 301},
        'DOWNLOADER_MIDDLEWARES': {
            'itkz.spider_ext.middlewares.m_agent.RandomUserAgentMiddleware': 542,
            'itkz.spider_ext.middlewares.m_proxy.ProxyMiddleware': 544,

            'itkz.spider_ext.middlewares.m_splash.RandomUserAgentMiddleware': 700,
            'itkz.spider_ext.middlewares.m_splash.SplashProxyMiddleware': 701,
            'scrapy_splash.SplashCookiesMiddleware': 723,
            'scrapy_splash.SplashMiddleware': 725,
            'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
        },

        'SPIDER_MIDDLEWARES': {'scrapy_splash.SplashDeduplicateArgsMiddleware': 100},
        'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter',
        'HTTPCACHE_STORAGE': 'scrapy_splash.SplashAwareFSCacheStorage',

        'SLEEP_TIME': 1,
        'DOWNLOAD_DELAY': 0.1,
        'DOWNLOAD_TIMEOUT': 12,
        'IS_USE_PROXY': True,
        'IS_USE_AGENT': True,
    }
    # custom_settings = settings_util.get_custom_settings(
    #     {
    #         'ITEM_PIPELINES': {'itkz.spider_ext.pipelines.itkz.ItkzDb': 301},
    #         'IS_USE_PROXY': True,
    #         'IS_USE_AGENT': True,
    #     }
    # )

    # 抓取的起始日期
    DATE_START_STR = '2021-08-01'
    DATE_START = 0

    # 列表模板
    URL_LIST_TEMPLATE = "https://itkz.net/api/goods?current={page}&categoryId=&labelId=&keyword="

    def init_child(self, kwargs):
        self.DATE_START = datetime.datetime.strptime(self.DATE_START_STR[:10], '%Y-%m-%d')

    def get_url(self, page):
        return self.URL_LIST_TEMPLATE.format(page=page)

    def parse(self, response):
        temp_data = eval(response.text).get('data')
        if not temp_data:
            self.send_log(f"返回内容为空，跳过：{response.text}\n{response.url}\n")
            return
        self.max_page = (temp_data.get('pages') or 1740) + self.BATCH
        for item in temp_data.get('records') or []:
            temp_item = ItkzItem()
            temp_item['i_dataId'] = item.get('id') or ''
            temp_item['i_categoryId'] = item.get('categoryId') or ''
            temp_item['i_categoryName'] = self.trim(item.get('categoryName'))[:20]
            temp_item['i_labelId'] = item.get('labelId') or ''
            temp_item['i_labelName'] = self.trim(item.get('labelName'))[:20]
            temp_item['i_cover'] = item.get('cover') or ''
            temp_item['i_title'] = self.trim(item.get('title'))[:100]
            temp_item['i_desc'] = self.trim(f"{item.get('keywords') or ''}||{item.get('description') or ''}||{item.get('content') or ''}")[:10000]
            temp_item['i_format'] = item.get('fileFormat') or ''
            temp_item['i_date'] = item.get('createTime') or ''
            temp_item['i_title_res'] = (item.get('title') or '')[:100]
            temp_item['i_keywords_res'] = (item.get('keywords') or '')[:200]
            temp_item['i_description_res'] = (item.get('description') or '')[:9000]
            temp_item['i_content_res'] = (item.get('content') or '')[:500]
            temp_item['i_paratext_res'] = (item.get('paratext') or '')[:200]
            yield temp_item
            if (datetime.datetime.strptime(temp_item['i_date'][:10], '%Y-%m-%d') - self.DATE_START).days <= 0:
                self.send_log(f"当前抓取日期【{temp_item['i_date']}】与预期时间【{self.DATE_START_STR}】不符合，退出爬虫，抓取完毕")
                self.force_exit_loop = True
                break

    def trim(self, v):
        """
        字符串清理
        :param v:
        :return:
        """
        v = v or ''
        for key in self.IGNORE_KEYS:
            v = v.replace(key, '')
        return str_util.sql_value_filter(v)
